#include "SplitToolCppJieba.h"

SplitToolCppJieba::SplitToolCppJieba()
: _dict_path("dict/jieba.dict.utf8")
, _model_path("dict/hmm_model.utf8")
, _user_dict_path("dict/user.dict.utf8")
, _idf_path("dict/idf.utf8")
, _stop_word_path("dict/stop_words.utf8")
, _jieba(_dict_path
         , _model_path
         , _user_dict_path
         , _idf_path
         , _stop_word_path)
{
    //const char* dict_path = "../dict/jieba.dict.utf8";
    //const char* model_path = "../dict/hmm_model.utf8";
    //const char* user_dict_path = "../dict/user.dict.utf8";
    //const char* idf_path = "../dict/idf.utf8";
    //const char* stop_word_path = "../dict/stop_words.utf8r;
}

SplitToolCppJieba::~SplitToolCppJieba()
{}

unordered_set<string> SplitToolCppJieba::StopWords(const string & filename)
{
    unordered_set<string> stopWords;
    ifstream ifs;
    ifs.open(filename);
    if(!ifs.is_open())
    {
        cerr << "打开文件失败: " << filename << "\n";
    }

    string word;
    while(getline(ifs, word))
    {
        word.erase(0,word.find_first_not_of(" \t\n\r"));
        word.erase(word.find_last_not_of(" \t\r\n") + 1);
        stopWords.insert(word);
    }

    return stopWords;
}

string SplitToolCppJieba::tokenize(const string & text)
{
    string sentence;

    for (char c : text)
    {
        if (c != '\r' && c != '\n' && c != ' ' && c != '\t')
        {
            sentence += c;
        }
    }

    return sentence;
}

vector<string> SplitToolCppJieba::cut(const string & filename)
{
    static unordered_set<string> cnStopWords = StopWords("Cn_yuliao/stop_words_zh.txt");
    vector<string> AllWords;
    ifstream file;
    file.open(filename);
    if(!file.is_open())
    {
        cerr << "打开文件失败：" << filename << "\n";
        return AllWords;
    }

    string line;
    vector<string> words;
    while(getline(file, line))
    {
        string sentence = tokenize(line);
        _jieba.Cut(sentence, words, true);

        // 使用过滤的方式减少多余操作
        vector<string> filteredWords;
        for (const auto& word : words)
        {
            if (cnStopWords.find(word) == cnStopWords.end())
            {
                filteredWords.push_back(word);
            }
        }
        AllWords.insert(AllWords.end(), filteredWords.begin(), filteredWords.end());
        words.clear();
    }

    file.close();
    return AllWords;
}
